package com.chance.cc.crawler.development.bootstrap.wangyiCar;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import com.chance.cc.crawler.development.scripts.allfeild.AICCommonField;
import org.apache.commons.lang3.StringUtils;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;

public class WangYiCarStart {
    private static final String domain = "wangyi";
    public static final String site = "carSeries";

    private static Proxy proxy = new Proxy();

    static {
//        代理配置
//        H5168QRFNIU3804D
//        5F6B3610BB719FAA
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }

    public static void main(String[] args) {

        CrawlerRequestRecord crawlerRequestRecord=autoAgeComment();//汽车时代评论


        DevCrawlerController devCrawlerController = DevCrawlerController.builder()
                .triggerInfo(domain, domain + "_trigger", System.currentTimeMillis(), domain + "_job")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain))
                .consoleResultPipeline("kafka")
                .consoleResultPipeline("redis")
                .fileResultPipeline("kafka","D:\\chance_log\\网易Kafka-8-3_1.log",false)
                .fileResultPipeline("redis","D:\\chance_log\\网易redis-8-3_1.log",false)
                .crawlerThreadNum(10)
                .requestRecord(crawlerRequestRecord)
                .build("com.chance.cc.crawler.development.scripts.wangyi");
        devCrawlerController.start();
    }

    //    汽车时代评论
    public static CrawlerRequestRecord autoAgeComment(){

        String url = "https://auto.163.com";
        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(domain)
                .httpUrl(url)
                .recordKey(url)
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*7, null))
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                .proxy(proxy)
                .build();
        crawlerRequestRecord.tagsCreator().bizTags().addDomain(domain);
        crawlerRequestRecord.tagsCreator().bizTags().addSite(site);
//        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Tag_Site_Info,"购 车");//CarBuy
//        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Tag_Site_Info,"新 车");//CarNew
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Tag_Site_Info,"试 驾");//CarDrive
//        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Tag_Site_Info,"导 购");//CarGuide
//        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Tag_Site_Info,"新能源");//CarEnergy
//        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Tag_Site_Info,"行 业");//CarIndustry

        //添加评论去重信息
        CrawlerRecord filterCrawlerRecord = new CrawlerRecord();//过滤爬虫记录
        filterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange);//根据key和时间过滤
        filterCrawlerRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-", filter, domain, "comment")));//内存过滤
        filterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24*7, null));//时间范围过滤
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(filterCrawlerRecord));//自定义标签

        return crawlerRequestRecord;
    }

}
